#-------------------------------------------------------------------------------
# Name:        scriptParser.py
# Purpose:     extract Wiki-to-Speech sequence from script text
#
# Author:      John Graves
#
# Created:     17 April 2011
# Modified:    15 February 2012
# Copyright:   (c) John Graves 2011, 2012
# Licence:     MIT license
#-------------------------------------------------------------------------------
from BeautifulSoup import BeautifulSoup
import codecs
import htmlentitydefs
import objects
import os
import re
import subprocess
import cherrypy
import sys
import urllib
from time import localtime, strftime

def unescape(text):
    def fixup(m):
        text = m.group(0)
        if text[:2] == "&#":
            # character reference
            try:
                if text[:3] == "&#x":
                    return unichr(int(text[3:-1], 16))
                else:
                    return unichr(int(text[2:-1]))
            except ValueError:
                pass
        else:
            # named entity
            try:
                text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
            except KeyError:
                pass
        return text # leave as is
    return re.sub("&#?\w+;", fixup, text)

def parseScript(name):
    if name.startswith("http"):
        if name.find("wikitospeech.pagekite.me")>0:
            sequence = parseForm(name)
        elif name.find("titanpad")>0:
            sequence = parseEtherpad(name)
        else:
            sequence = parseHtml(name)
    else:
        if name.endswith(".txt"):
            sequence = parseTxtFile(name)
        else:
            sequence = None
##            # No parsing of .html
##            # Set up dummy sequence
##            seq = objects.Sequence()
##            seq.sequence = []
##            question = objects.Question()
##            question.questionTexts = []
##            question.questionTexts.append("Opening "+name)
##            question.linkToShow = name
##            seq.sequence.append(question)
##            sequence = seq.sequence
##            dumpSequence(seq, False)
    return sequence

def parseEtherpad(name):
    # download Etherpad and extract script
    # http://ietherpad.com/mieeiphS5# J
    try:
        proxy = os.environ["HTTP_PROXY"]
    except:
        proxy = ''
    if proxy=="http://cache.aut.ac.nz:3128":
        proxies = {'http': 'http://cache.aut.ac.nz:3128'}
        urlOpen = urllib.urlopen( name , proxies=proxies )
    else:
        urlOpen = urllib.urlopen( name, proxies={} )
    soup = BeautifulSoup(urlOpen.read())
    postbody = soup.find("div", { "class" : "postbody" })
    soupString = ""
    try:
        soupString = str(soup)
    except UnicodeDecodeError:
        # no luck here, give up
        return None
    if len(soupString) > 0:
        cleanUnicodeTextStr = \
        soupString[ soupString.find(u'"initialAttributedText":{"text"')+33 : \
                   soupString.find(u',"attribs":')-1 ]
        text = unescape(cleanUnicodeTextStr).split('\\n')
    else:
        return None
    sequence = parseText(text)
    return sequence

def parseForm(name):
    # download and extract script from wikitospeech.pagekite.me page, for example:
    # http://wikitospeech.pagekite.me?tag=Test
    try:
        proxy = os.environ["HTTP_PROXY"]
    except:
        proxy = ''
    if proxy=="http://cache.aut.ac.nz:3128":
        proxies = {'http': 'http://cache.aut.ac.nz:3128'}
        urlOpen = urllib.urlopen( name , proxies=proxies )
    else:
        urlOpen = urllib.urlopen( name, proxies={} )

    # extract text marked with <from> from wikitospeech page
    soup = BeautifulSoup(urlOpen.read())
    taggedPre = soup.form
    img=[i['src'] for i in soup.form.findAll('img')]
    #[u'http://dl.dropbox.com/u/12838403/20120109/img0.png', u'http://dl.dropbox.com/u/12838403/20120109/img1.png']
    text=[i.contents[0].strip() for i in soup.form.findAll('textarea')]
    #[u'Notes on slide 1', u'Notes on slide 2']
    urlText = []
    for i in range(0,len(img)):
        urlText.append(img[i])
        textLines = text[i].split("\n")
        for line in textLines:
            urlText.append(line)
        urlText.append("\n")

    f = open('debug.txt','w')
    f.write(u"test run at " + strftime("%d %b %Y %H:%M", localtime()) + "\n")
    f.write(str(type(urlText))+ "\n")
    f.write(str(len(urlText))+ "\n")
    for l in urlText:
        f.write(l.strip()+"\n")
    f.close()

    sequence = parseText(urlText)
    return sequence

def parseHtml(name):
    # download and extract script from wiki page, for example:
    # http://dl.dropbox.com/u/12838403/dropbox.txt
    try:
        proxy = os.environ["HTTP_PROXY"]
    except:
        proxy = ''
    try:
        if proxy=="http://cache.aut.ac.nz:3128":
            proxies = {'http': 'http://cache.aut.ac.nz:3128'}
            urlOpen = urllib.urlopen( name , proxies=proxies )
        else:
            urlOpen = urllib.urlopen( name, proxies={} )
    except:
        return None
    if name.endswith(".txt"):
        urlText = urlOpen.read().split("\n")
    else:
        # extract text marked with <pre> from wiki or blog page
        soup = BeautifulSoup(urlOpen.read())
        taggedPre = soup.pre
        if taggedPre != None:
            taggedPreStr = str(taggedPre).replace('<br />\n','\n')
            taggedPreStr = taggedPreStr.replace('<br />','\n')
            soup = BeautifulSoup(taggedPreStr)
            urlText = unescape(''.join(soup.pre.findAll(text=True))).splitlines()
        else:
            return None

    f = open('debug.txt','w')
    f.write(u"test run at " + strftime("%d %b %Y %H:%M", localtime()) + "\n")
    f.write(str(type(urlText))+ "\n")
    f.write(str(len(urlText))+ "\n")
    for l in urlText:
        f.write(l.strip()+"\n")
    f.close()

    sequence = parseText(urlText)
    return sequence

def parseTxtFile(text):
##    # open txt file
##    try:
##        f = codecs.open(name, encoding='utf-8', mode='r')
##    except:
##        # No parsing of .txt
##        return None
##    text = f.readlines()
##    return str(name)

#     if not sys.platform.startswith("win"):
#         # find slide images in .txt file and make symbolic links if possible
#         pngs = [item.strip() for item in text if item.endswith(".png\n")]
#         # script source directory
#         scriptDir, scriptFile = os.path.split(name)
#         savePath = os.getcwd()
#         os.chdir('static')
#         staticDir = os.listdir(".")
#         pngsInStatic = [file for file in staticDir if file.endswith(".png")]
#         for png in pngs:
#             if png in pngsInStatic:
#                 subprocess.Popen(["rm",png])
#             subprocess.Popen(["ln","-s",scriptDir+os.sep+png,png])
#         os.chdir(savePath)

    sequence = parseText(text)
    return sequence

def parseText(text):
    # guess at questionMode based on first line
    if text[0].lower().endswith(u".png") or text[0].startswith(u"[path="):
        questionMode = False
    else:
        questionMode = True
    seq = objects.Sequence()
    seq.sequence = []
    pathToImageFiles = ""
    question = objects.Question()
    answer = objects.Answer()
    rule = objects.Rule()
    ruleMode = False
    for line in text:
        line = line.strip()
        question.unparsedQuestion += line +"\n"
        if not line.startswith(u"#"): # ignore comment lines
            if (line.startswith(u"[") and not line.endswith(";")):
                if len(question.questionTexts)>0:
                    seq.sequence.append(question)
                    question = objects.Question()
                    question.pathToImageFiles = pathToImageFiles
                equalsAt = line.find(u"=")
                if equalsAt>1: # set parameters
                    parameterName = line[1:equalsAt].strip().lower()
                    parameterValue = line[equalsAt+1:line.find(u"]")].strip().lower()
                    if parameterName == u"questions":
                        if parameterValue == u"on":
                            questionMode = True
                        if parameterValue == u"off":
                            questionMode = False
                            ruleMode = False
                        continue
                    elif parameterName == u"path":
                        if((not parameterValue.endswith(os.sep) and (not parameterValue.endswith("\\")))):
                            parameterValue += os.sep
                        pathToImageFiles = parameterValue
                        question.pathToImageFiles = pathToImageFiles
                        continue
                else:
                    if line == u"[_link]":
                        ruleMode = True
                    if ruleMode:
                        # rule names have two leading square brackets: [[name]]
                        rule.ruleName = line[2:line.find(u"]")]
                        rule.ruleType = u"link"
                    else:
                        # question tag 
                        question.tag = line[1:line.find(u"]")]

            else:
                line = line.strip()
                if questionMode == False:
                    # In default mode, expected arrangement of links and text
                    # is
                    #    LINK (ending in http/html/jpg)
                    #    TEXT FOR VOICE OVER
                    #    (optionally several lines long)
                    #
                    if len(line)>0:
                        if (line.startswith(u"http") or
                            line.endswith(u".html") or
                            line.endswith(u".jpg") or
                            line.endswith(u".JPG") or
                            line.endswith(u".png") or
                            line.endswith(u".PNG")):
                                if len(question.questionTexts)>0:
                                    seq.sequence.append(question)
                                    question = objects.Question()
                                    question.pathToImageFiles = pathToImageFiles
                                question.linkToShow = line
                        else: 
                            question.questionTexts.append(line)
                else:                
                    # questionMode == True
                    # In question mode, expected arrangement
                    # is
                    #    (optional image to show)
                    #    QUESTION
                    #    (optionally several lines long)
                    #    [LINK ANSWER] ; [ACTION] [RESPONSE]
                    #    (optionally repeated)
                    #
                    # (unless ALSO in rule mode)
                    if len(line)>0:
                        if ruleMode == True:
                            # parse line for rule
                            (ruleAttribute, ruleAttributeValue) = line.split("=")
                            ruleAttribute = ruleAttribute.strip() 
                            ruleAttributeValue = ruleAttributeValue.strip()
                            print (ruleAttribute, ruleAttributeValue)
                            # a regular expression overrides an example
                            if u're'==ruleAttribute:
                                rule.rulePattern = re.compile(ruleAttributeValue, re.IGNORECASE)
                            elif u'reply'==ruleAttribute:
                                rule.ruleResponse = ruleAttributeValue
                            elif u'example'==ruleAttribute:
                                # turn example into a regular expression
                                openBracketAt = ruleAttributeValue.find(u'[')
                                closeBracketAt = ruleAttributeValue.find(u']', openBracketAt + 1)
                                secondOpenBracketAt = ruleAttributeValue.find(u'[', closeBracketAt + 1)
                                secondCloseBracketAt = ruleAttributeValue.find(u']', secondOpenBracketAt + 1)
                                if openBracketAt == -1:
                                    # if no brackets, use whole example as the regular expression
                                    rule.rulePattern = re.compile(ruleAttributeValue, re.IGNORECASE)
                                else:
                                    firstPartRE = ruleAttributeValue[ openBracketAt + 1 : closeBracketAt ].strip()
                                    rule.rulePattern = '(' + firstPartRE + ')(.*)'
                                    if openBracketAt > 0:
                                        rule.rulePattern = '(.*)' + rule.rulePattern
                                    if secondOpenBracketAt > closeBracketAt:
                                        secondPartRE = ruleAttributeValue[ secondOpenBracketAt + 1 : secondCloseBracketAt ].strip()
                                        rule.rulePattern = rule.rulePattern + '(' + secondPartRE + ')(.*)'
                                    rule.rulePattern = re.compile(rule.rulePattern, re.IGNORECASE)
                        else:
                            # Not in rule mode
                            if (line.endswith(u".jpg") or
                                line.endswith(u".JPG") or
                                line.endswith(u".png") or
                                line.endswith(u".PNG")):
                                    if len(question.questionTexts)>0:
                                        seq.sequence.append(question)
                                        question = objects.Question()
                                        question.pathToImageFiles = pathToImageFiles
                                    question.linkToShow = line
                            else:
                                if line == u"[input];":
                                    answer.input = True
                                semicolonAt = line.find(u";")
                                if -1 == semicolonAt:
                                    question.questionTexts.append(line)
                                else: # parse answer
                                    answer.answerText = line[0:semicolonAt].strip()
                                    if line.startswith(u"[http:"):
                                        spaceAt = line.find(u" ")
                                        if 1<spaceAt and spaceAt+1<len(line):
                                            answer.answerSideLink = line[1:spaceAt]
                                            answer.answerText = line[spaceAt+1:line.find(u"]")]
                                
                                    responseSide = line[semicolonAt+1:].strip()
                                    if len(responseSide)>0:
                                        while responseSide.startswith(u";"):
                                            answer.action += 1
                                            responseSide = responseSide[1:].strip()
                                        if responseSide.startswith(u"["):
                                            rightbracketAt = responseSide.find(u"]")
                                            # default
                                            answer.responseSideLink = responseSide[1:rightbracketAt]
                                            responseSide = responseSide[rightbracketAt+1:].strip()
                                            # check for [[next script]]
                                            if answer.responseSideLink.startswith(u"["):
                                                nextScript = answer.responseSideLink[1:]
                                                if responseSide.startswith(u"]"):
                                                    responseSide = responseSide[1:].strip()
                                                # turn name of next script into fetch
                                                tagForURL = urllib.urlencode({'tag':nextScript})
                                                answer.responseSideLink="/fetchScriptToView?"+tagForURL
                                        answer.responseText = responseSide
                                    if answer.answerText == u"[next]":
                                        answer.action = 1
                                    question.answers.append(answer)
                                    answer = objects.Answer()

                    else: # blank line
                        if len(question.questionTexts)>0:
                            seq.sequence.append(question)
                            question = objects.Question()
                            question.pathToImageFiles = pathToImageFiles
                        if len(rule.ruleName)>0:
                            seq.rules.append(rule)
                            rule = objects.Rule()

    if len(question.questionTexts)>0:
        seq.sequence.append(question)
    if len(rule.ruleName)>0:
        seq.rules.append(rule)

    # second pass to match responseSideLinks to tags and adjust actions
    tags = [ question.tag.lower() for question in seq.sequence ]
    for qnum, question in enumerate( seq.sequence ):
        for answer in question.answers:
            if answer.responseSideLink != "":
                if answer.responseSideLink.lower() in tags:
                    # remove link
                    answer.action = tags.index(answer.responseSideLink.lower()) - qnum
                    answer.responseSideLink = ""

    dumpSequence(seq, questionMode)

    return seq

def dumpSequence(seq, questionMode):
    if True:
        f = open('output.json','w')
        f.write("""{"Q":
    [
        {""")
        #f.write("test run at " + strftime("%d %b %Y %H:%M", localtime()))
        #f.write("\nquestionMode is "+ str(questionMode))
        for i, q in enumerate(seq.sequence):
            #f.write("\nQuestion "+str(i)+"-"*30)
            f.write('\n            "t":"'+q.tag+'",')
            f.write('\n            "l":"en",')
            #f.write("\n  pathToImageFiles: "+q.pathToImageFiles)
            qtext = " ".join(q.questionTexts)
            f.write('\n            "q":"'+qtext+'",')
            f.write('\n            "i":"'+q.linkToShow+'",')
            if len(q.answers)==0:
                f.write('\n            "A": []')
            else:
                f.write('\n            "A": [')
                for j, a in enumerate(q.answers):
                    # write a new line if this is the first item
                    if j==0:
                        f.write('\n                    ')
                    
                    #write opening brace for answer object
                    f.write('{')
                    
                    # write answer object content
                    f.write('\n                        "a":"'+a.answerText+'",')
                    f.write('\n                        "s":"'+a.answerSideLink+'",')
                    f.write('\n                        "l":"en",')
                    f.write('\n                        "m":'+str(a.action)+',')
                    f.write('\n                        "r":"'+a.responseText+'",')
                    f.write('\n                        "k":"'+a.responseSideLink+'",')
                    f.write('\n                        "f":"en",')
                    f.write('\n                        "i":'+str(a.input).lower())
                    
                    # write closing brace for answer object
                    f.write('\n                    }') 
                    
                    # write a comma if this isn't the last item 
                    if j < len(q.answers)-1:
                        f.write(',')
                # write closing brace for this answer
                f.write('\n                 ]')
            # Supply braces for Question            
            if i<len(seq.sequence)-1:
                f.write('\n                    },{')
            #for l in q.questionTexts:
            #    f.write("\n      questionText: "+l)
            #for j, a in enumerate(q.answers):
            #    f.write("\n       answerText"+str(j)+": "+a.answerText)
            #    f.write("\n   answerSideLink"+str(j)+": "+a.answerSideLink)
            #    f.write("\n responseSideLink"+str(j)+": "+a.responseSideLink)
            #    f.write("\n     responseText"+str(j)+": "+a.responseText)
            #    f.write("\n           action"+str(j)+": "+str(a.action))
            #    f.write("\n            input"+str(j)+": "+str(a.input))
            #f.write("\n unparsedQuestion"+str(i)+": "+str(q.unparsedQuestion))
        #for i, r in enumerate(seq.rules):
        #    f.write("\nRule "+str(i)+"-"*30)
        #    f.write("\n              name: "+r.ruleName)
        #    f.write("\n           pattern: "+str(r.rulePattern))
        #    f.write("\n              type: "+r.ruleType)
        #    f.write("\n          response: "+r.ruleResponse)
        
        f.write("""\n        }
    ]
}\n""")
        f.close()

if __name__ == "__main__":
    #f = open('/Users/johngraves/20120123/SlideSpeechConverterWeb/static/worktheroom/script.txt','r')
    f = open('test.txt','r')
    text = f.readlines()
    parseText(text)
    #parseHtml("http://dl.dropbox.com/u/12838403/20111123/SurfersGoneWild/script.txt")
